﻿//using System;
//using System.Collections.Concurrent;
//using System.Collections.Generic;
//using System.Linq;
//using System.Net.Http;
//using System.Text.RegularExpressions;
//using System.Threading.Tasks;

//namespace HW_g
//{
//    class Program
//    {
//        static async Task Main(string[] args)
//        {
//            Console.WriteLine("请输入搜索关键字：");
//            string keyword = Console.ReadLine();

//            // 使用搜索引擎生成搜索 URL
//            string searchEngineUrl = $"https://www.bing.com/search?q={Uri.EscapeDataString(keyword)}";

//            // 爬取搜索结果页面
//            var urls = await GetSearchResultUrls(searchEngineUrl);

//            // 爬取电话号码
//            var phoneNumbers = await CrawlPhoneNumbers(urls);

//            // 显示结果
//            Console.WriteLine("\n已爬取的URL：");
//            foreach (var url in urls)
//            {
//                Console.WriteLine(url);
//            }

//            Console.WriteLine("\n找到的电话号码及其来源URL：");
//            foreach (var phone in phoneNumbers)
//            {
//                Console.WriteLine($"电话号码: {phone.Key}");
//                Console.WriteLine("来源URL:");
//                foreach (var url in phone.Value)
//                {
//                    Console.WriteLine($"  - {url}");
//                }
//            }
//        }

//        // 获取搜索结果中的 URL
//        private static async Task<List<string>> GetSearchResultUrls(string searchEngineUrl)
//        {
//            List<string> urls = new List<string>();
//            try
//            {
//                using (HttpClient client = new HttpClient())
//                {
//                    string html = await client.GetStringAsync(searchEngineUrl);

//                    // 使用正则表达式提取搜索结果中的 URL
//                    Regex regex = new Regex(@"https?://[^\s""<>]+");
//                    MatchCollection matches = regex.Matches(html);

//                    foreach (Match match in matches)
//                    {
//                        string url = match.Value;
//                        if (!urls.Contains(url))
//                        {
//                            urls.Add(url);
//                        }
//                    }
//                }
//            }
//            catch (Exception ex)
//            {
//                Console.WriteLine($"获取搜索结果失败: {ex.Message}");
//            }
//            return urls.Take(50).ToList(); // 限制为最多 50 个 URL
//        }

//        // 爬取电话号码
//        private static async Task<ConcurrentDictionary<string, List<string>>> CrawlPhoneNumbers(List<string> urls)
//        {
//            ConcurrentDictionary<string, List<string>> phoneNumbers = new ConcurrentDictionary<string, List<string>>();
//            List<Task> tasks = new List<Task>();

//            foreach (var url in urls)
//            {
//                tasks.Add(Task.Run(async () =>
//                {
//                    try
//                    {
//                        using (HttpClient client = new HttpClient())
//                        {
//                            string html = await client.GetStringAsync(url);

//                            // 使用正则表达式提取电话号码
//                            Regex regex = new Regex(@"\b\d{3,4}[-.\s]?\d{7,8}\b");
//                            MatchCollection matches = regex.Matches(html);

//                            foreach (Match match in matches)
//                            {
//                                string phone = match.Value;
//                                phoneNumbers.AddOrUpdate(phone,
//                                    new List<string> { url },
//                                    (key, existingUrls) =>
//                                    {
//                                        if (!existingUrls.Contains(url))
//                                        {
//                                            existingUrls.Add(url);
//                                        }
//                                        return existingUrls;
//                                    });
//                            }
//                        }
//                    }
//                    catch (Exception ex)
//                    {
//                        Console.WriteLine($"爬取 {url} 时出错: {ex.Message}");
//                    }
//                }));
//            }

//            await Task.WhenAll(tasks);

//            // 限制为最多 100 个电话号码
//            return new ConcurrentDictionary<string, List<string>>(phoneNumbers.Take(100));
//        }
//    }
//}


using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using HtmlAgilityPack;

namespace HW_g
{
    class Program
    {
        static async Task Main(string[] args)
        {
            Console.WriteLine("请输入搜索关键字：");
            string keyword = Console.ReadLine();

            // 多页 Bing 搜索（第一页 + 第二页）
            var urls = new List<string>();
            for (int i = 0; i < 2; i++)
            {
                int offset = i * 10 + 1;
                string searchUrl = $"https://www.bing.com/search?q={Uri.EscapeDataString(keyword)}&first={offset}";
                var resultUrls = await GetSearchResultUrls(searchUrl);
                urls.AddRange(resultUrls);
            }
            urls = urls.Distinct().ToList();

            var phoneNumbers = await CrawlPhoneNumbers(urls);

            Console.WriteLine("\n已爬取的URL：");
            foreach (var url in urls)
            {
                Console.WriteLine(url);
            }

            Console.WriteLine("\n找到的电话号码及其来源URL：");
            foreach (var phone in phoneNumbers)
            {
                Console.WriteLine($"电话号码: {phone.Key}");
                foreach (var url in phone.Value)
                {
                    Console.WriteLine($"  - {url}");
                }
            }
        }

        // 使用 HtmlAgilityPack 提取搜索结果中真实网址
        private static async Task<List<string>> GetSearchResultUrls(string searchEngineUrl)
        {
            List<string> urls = new List<string>();
            try
            {
                using (HttpClient client = new HttpClient())
                {
                    client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64)");
                    string html = await client.GetStringAsync(searchEngineUrl);

                    HtmlDocument doc = new HtmlDocument();
                    doc.LoadHtml(html);

                    var nodes = doc.DocumentNode.SelectNodes("//li[@class='b_algo']//h2/a");

                    if (nodes != null)
                    {
                        foreach (var node in nodes)
                        {
                            string href = node.GetAttributeValue("href", "");
                            if (!string.IsNullOrEmpty(href) && href.StartsWith("http"))
                            {
                                urls.Add(href);
                            }
                        }
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine($"搜索结果解析失败: {ex.Message}");
            }
            return urls;
        }

        // 并发爬取电话，限制最大并发为5
        private static async Task<ConcurrentDictionary<string, List<string>>> CrawlPhoneNumbers(List<string> urls)
        {
            ConcurrentDictionary<string, List<string>> phoneNumbers = new ConcurrentDictionary<string, List<string>>();
            List<Task> tasks = new List<Task>();
            SemaphoreSlim semaphore = new SemaphoreSlim(5); // 限制最大并发为5

            foreach (var url in urls)
            {
                tasks.Add(Task.Run(async () =>
                {
                    await semaphore.WaitAsync();
                    try
                    {
                        using (HttpClient client = new HttpClient())
                        {
                            client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (Windows NT 10.0; Win64; x64)");
                            string html = await client.GetStringAsync(url);

                            // 更强的电话正则：支持 +86、区号、横杠、小括号等
                            Regex regex = new Regex(@"\b(?:\+?\d{1,4}[-.\s]?)?(?:\(?\d{2,4}\)?[-.\s]?)?\d{7,8}\b");
                            MatchCollection matches = regex.Matches(html);

                            foreach (Match match in matches)
                            {
                                if (phoneNumbers.Count >= 100) break;

                                string phone = match.Value;
                                phoneNumbers.AddOrUpdate(phone,
                                    new List<string> { url },
                                    (key, existingUrls) =>
                                    {
                                        if (!existingUrls.Contains(url))
                                        {
                                            existingUrls.Add(url);
                                        }
                                        return existingUrls;
                                    });
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        Console.WriteLine($"爬取 {url} 出错: {ex.Message}");
                    }
                    finally
                    {
                        semaphore.Release();
                    }
                }));
            }

            await Task.WhenAll(tasks);

            // 限制最多 100 个
            return new ConcurrentDictionary<string, List<string>>(phoneNumbers.Take(100));
        }
    }
}
